{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5e923fd1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-05T23:04:54.529272Z",
     "iopub.status.busy": "2024-10-05T23:04:54.528398Z",
     "iopub.status.idle": "2024-10-05T23:05:11.577833Z",
     "shell.execute_reply": "2024-10-05T23:05:11.576751Z"
    },
    "papermill": {
     "duration": 17.059171,
     "end_time": "2024-10-05T23:05:11.580248",
     "exception": false,
     "start_time": "2024-10-05T23:04:54.521077",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\r\n",
      "bigframes 0.22.0 requires google-cloud-bigquery[bqstorage,pandas]>=3.10.0, but you have google-cloud-bigquery 2.34.4 which is incompatible.\r\n",
      "bigframes 0.22.0 requires google-cloud-storage>=2.0.0, but you have google-cloud-storage 1.44.0 which is incompatible.\r\n",
      "bigframes 0.22.0 requires pandas<2.1.4,>=1.5.0, but you have pandas 2.2.2 which is incompatible.\r\n",
      "cesium 0.12.3 requires numpy<3.0,>=2.0, but you have numpy 1.26.4 which is incompatible.\r\n",
      "dataproc-jupyter-plugin 0.1.79 requires pydantic~=1.10.0, but you have pydantic 2.9.2 which is incompatible.\u001b[0m\u001b[31m\r\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "!pip install scikit-learn -U -q"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "68d52e5b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-05T23:05:11.592607Z",
     "iopub.status.busy": "2024-10-05T23:05:11.592250Z",
     "iopub.status.idle": "2024-10-05T23:05:13.812026Z",
     "shell.execute_reply": "2024-10-05T23:05:13.810937Z"
    },
    "papermill": {
     "duration": 2.228606,
     "end_time": "2024-10-05T23:05:13.814509",
     "exception": false,
     "start_time": "2024-10-05T23:05:11.585903",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Importing core libraries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import shutil\n",
    "from time import time\n",
    "import pprint\n",
    "import joblib\n",
    "from functools import partial\n",
    "import gc\n",
    "import os\n",
    "# Suppressing warnings because of skopt verbosity\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "# Classifier/Regressor\n",
    "from xgboost import XGBRegressor\n",
    "\n",
    "# Model selection\n",
    "from sklearn.model_selection import KFold, StratifiedKFold\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# Metrics\n",
    "from sklearn.metrics import root_mean_squared_error\n",
    "from sklearn.metrics import make_scorer\n",
    "\n",
    "# Skopt functions\n",
    "from skopt import BayesSearchCV\n",
    "from skopt.callbacks import DeadlineStopper, DeltaYStopper\n",
    "from skopt.space import Real, Categorical, Integer\n",
    "from skopt.utils import use_named_args # decorator to convert a list of parameters to named arguments\n",
    "from skopt import gp_minimize, forest_minimize, gbrt_minimize, dummy_minimize\n",
    "\n",
    "# Data processing\n",
    "from sklearn.preprocessing import OrdinalEncoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6c4498ef",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-05T23:05:13.826652Z",
     "iopub.status.busy": "2024-10-05T23:05:13.826188Z",
     "iopub.status.idle": "2024-10-05T23:05:18.689174Z",
     "shell.execute_reply": "2024-10-05T23:05:18.687887Z"
    },
    "papermill": {
     "duration": 4.872322,
     "end_time": "2024-10-05T23:05:18.692283",
     "exception": false,
     "start_time": "2024-10-05T23:05:13.819961",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Loading data \n",
    "X_train = pd.read_csv(\"../input/30-days-of-ml/train.csv\")\n",
    "X_test = pd.read_csv(\"../input/30-days-of-ml/test.csv\")\n",
    "\n",
    "# Preparing data as a tabular matrix\n",
    "X_train = X_train.set_index('id')\n",
    "X_test = X_test.set_index('id')\n",
    "\n",
    "# Pointing out categorical features\n",
    "categoricals = [item for item in X_train.columns if 'cat' in item]\n",
    "\n",
    "# Dealing with categorical data using OrdinalEncoder\n",
    "ordinal_encoder = OrdinalEncoder()\n",
    "X_train[categoricals] = ordinal_encoder.fit_transform(X_train[categoricals]).astype(int)\n",
    "X_test[categoricals] = ordinal_encoder.transform(X_test[categoricals]).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "9fb868ba",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-05T23:05:18.704310Z",
     "iopub.status.busy": "2024-10-05T23:05:18.703971Z",
     "iopub.status.idle": "2024-10-05T23:05:18.708073Z",
     "shell.execute_reply": "2024-10-05T23:05:18.707339Z"
    },
    "papermill": {
     "duration": 0.012352,
     "end_time": "2024-10-05T23:05:18.710032",
     "exception": false,
     "start_time": "2024-10-05T23:05:18.697680",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Setting the scoring function\n",
    "scoring = root_mean_squared_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "739acd60",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-05T23:05:18.722436Z",
     "iopub.status.busy": "2024-10-05T23:05:18.722130Z",
     "iopub.status.idle": "2024-10-05T23:05:18.726084Z",
     "shell.execute_reply": "2024-10-05T23:05:18.725334Z"
    },
    "papermill": {
     "duration": 0.011828,
     "end_time": "2024-10-05T23:05:18.728055",
     "exception": false,
     "start_time": "2024-10-05T23:05:18.716227",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Setting the cv strategy\n",
    "kf = KFold(n_splits=5, shuffle=True, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "16c9d682",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-05T23:05:18.739207Z",
     "iopub.status.busy": "2024-10-05T23:05:18.738918Z",
     "iopub.status.idle": "2024-10-05T23:05:30.895899Z",
     "shell.execute_reply": "2024-10-05T23:05:30.895016Z"
    },
    "papermill": {
     "duration": 12.165229,
     "end_time": "2024-10-05T23:05:30.898370",
     "exception": false,
     "start_time": "2024-10-05T23:05:18.733141",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cbdeaa37",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-05T23:05:30.910655Z",
     "iopub.status.busy": "2024-10-05T23:05:30.910068Z",
     "iopub.status.idle": "2024-10-05T23:05:30.916150Z",
     "shell.execute_reply": "2024-10-05T23:05:30.915240Z"
    },
    "papermill": {
     "duration": 0.014086,
     "end_time": "2024-10-05T23:05:30.918096",
     "exception": false,
     "start_time": "2024-10-05T23:05:30.904010",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def df_to_dataset(dataframe, shuffle=True, batch_size=32):\n",
    "    dataframe = dataframe.copy()\n",
    "    labels = dataframe.pop('target')\n",
    "    ds = tf.data.Dataset.from_tensor_slices((dict(dataframe), labels))\n",
    "    if shuffle:\n",
    "        ds = ds.shuffle(buffer_size=len(dataframe))\n",
    "    ds = ds.batch(batch_size)\n",
    "    return ds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "02641434",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-05T23:05:30.929396Z",
     "iopub.status.busy": "2024-10-05T23:05:30.929098Z",
     "iopub.status.idle": "2024-10-05T23:05:30.943273Z",
     "shell.execute_reply": "2024-10-05T23:05:30.942421Z"
    },
    "papermill": {
     "duration": 0.022002,
     "end_time": "2024-10-05T23:05:30.945253",
     "exception": false,
     "start_time": "2024-10-05T23:05:30.923251",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def create_model(cat0_dim, cat1_dim, cat2_dim,\n",
    "                 cat3_dim, cat4_dim, cat5_dim,\n",
    "                 cat6_dim, cat7_dim, cat8_dim, cat9_dim,\n",
    "                 layers, layer_1, layer_2, layer_3, layer_4, layer_5,\n",
    "                 activation, dropout, batch_normalization, learning_rate, **others):\n",
    "    \n",
    "    # Define name labels\n",
    "    categorical_labels = ['cat0', 'cat1', 'cat2', 'cat3', 'cat4', 'cat5',\n",
    "                          'cat6', 'cat7', 'cat8', 'cat9']\n",
    "    numeric_labels = ['cont1', 'cont2', 'cont3', 'cont4', 'cont5', 'cont6',\n",
    "                      'cont7', 'cont8', 'cont9', 'cont10', 'cont11', 'cont12', 'cont13']\n",
    "    \n",
    "    # Define dimensions for categorical features\n",
    "    dims = {'cat0': cat0_dim, 'cat1': cat1_dim, 'cat2': cat2_dim, \n",
    "            'cat3': cat3_dim, 'cat4': cat4_dim, 'cat5': cat5_dim,\n",
    "            'cat6': cat6_dim, 'cat7': cat7_dim, 'cat8': cat8_dim, 'cat9': cat9_dim}\n",
    "    \n",
    "    # Vocabulary based on unique categorical values in X_train\n",
    "    vocab = {h: X_train[h].max() for h in categorical_labels}\n",
    "    \n",
    "    layers = [layer_1, layer_2, layer_3, layer_4, layer_5][:layers]\n",
    "\n",
    "    # Input placeholders for all features (continuous and categorical)\n",
    "    continuous_inputs = []\n",
    "    categorical_inputs = []\n",
    "    flattened_categorical_inputs = []\n",
    "    \n",
    "    # Continuous feature inputs\n",
    "    for header in numeric_labels:\n",
    "        input_layer = tf.keras.Input(shape=(1,), name=header)\n",
    "        continuous_inputs.append(input_layer)\n",
    "\n",
    "    # Categorical feature inputs with embeddings\n",
    "    for header in categorical_labels:\n",
    "        input_layer = tf.keras.Input(shape=(1,), name=header)\n",
    "        embedded_input = tf.keras.layers.Embedding(input_dim=vocab[header] + 1, \n",
    "                                                   output_dim=dims[header], \n",
    "                                                   name=f'{header}_embedding')(input_layer)\n",
    "        flattened_input = tf.keras.layers.Flatten()(embedded_input)\n",
    "        categorical_inputs.append(input_layer)\n",
    "        flattened_categorical_inputs.append(flattened_input)\n",
    "\n",
    "    # Concatenate all inputs (continuous and categorical)\n",
    "    concatenated_inputs = tf.keras.layers.Concatenate()(continuous_inputs + flattened_categorical_inputs)\n",
    "        \n",
    "    # Build the dense network based on the provided architecture\n",
    "    x = concatenated_inputs\n",
    "    for nodes in layers:\n",
    "        x = tf.keras.layers.Dense(nodes, activation=activation)(x)\n",
    "        if batch_normalization:\n",
    "            x = tf.keras.layers.BatchNormalization()(x)\n",
    "        if dropout > 0:\n",
    "            x = tf.keras.layers.Dropout(dropout)(x)\n",
    "\n",
    "    # Final output layer\n",
    "    output = tf.keras.layers.Dense(1)(x)\n",
    "\n",
    "    # Define the complete model using Functional API\n",
    "    model = tf.keras.Model(inputs=continuous_inputs + categorical_inputs, outputs=output)\n",
    "\n",
    "    # Compile the model\n",
    "    model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=learning_rate),\n",
    "                  loss=tf.keras.losses.MeanSquaredError(),\n",
    "                  metrics=['root_mean_squared_error'])\n",
    "\n",
    "    return model\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "9ad38bf5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-05T23:05:30.956519Z",
     "iopub.status.busy": "2024-10-05T23:05:30.956244Z",
     "iopub.status.idle": "2024-10-05T23:05:31.041894Z",
     "shell.execute_reply": "2024-10-05T23:05:31.040990Z"
    },
    "papermill": {
     "duration": 0.093993,
     "end_time": "2024-10-05T23:05:31.044357",
     "exception": false,
     "start_time": "2024-10-05T23:05:30.950364",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Setting the search space\n",
    "\n",
    "from tensorflow.keras.activations import relu, leaky_relu\n",
    "\n",
    "space = [Integer(1, 2, name='cat0_dim'),\n",
    "         Integer(1, 2, name='cat1_dim'),\n",
    "         Integer(1, 2, name='cat2_dim'),\n",
    "         Integer(1, 3, name='cat3_dim'),\n",
    "         Integer(1, 3, name='cat4_dim'),\n",
    "         Integer(1, 3, name='cat5_dim'),\n",
    "         Integer(1, 4, name='cat6_dim'),\n",
    "         Integer(1, 4, name='cat7_dim'),\n",
    "         Integer(1, 6, name='cat8_dim'),\n",
    "         Integer(1, 8, name='cat9_dim'),\n",
    "         Integer(1, 5, name='layers'),\n",
    "         Integer(2, 256, name='layer_1'),\n",
    "         Integer(2, 256, name='layer_2'),\n",
    "         Integer(2, 256, name='layer_3'),\n",
    "         Integer(2, 256, name='layer_4'),\n",
    "         Integer(2, 256, name='layer_5'),\n",
    "         Categorical([leaky_relu, relu], name='activation'),\n",
    "         Real(0.0, 0.5, 'uniform', name='dropout'),\n",
    "         Categorical([True, False], name='batch_normalization'),\n",
    "         Categorical([0.01, 0.005, 0.002, 0.001], name='learning_rate'),\n",
    "         Integer(256, 1024, name='batch_size')\n",
    "        ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a4410f71",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-05T23:05:31.056042Z",
     "iopub.status.busy": "2024-10-05T23:05:31.055705Z",
     "iopub.status.idle": "2024-10-05T23:05:31.072400Z",
     "shell.execute_reply": "2024-10-05T23:05:31.071264Z"
    },
    "papermill": {
     "duration": 0.024808,
     "end_time": "2024-10-05T23:05:31.074408",
     "exception": false,
     "start_time": "2024-10-05T23:05:31.049600",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "counter = 0\n",
    "history = {i:list() for i in range(5)}\n",
    "used_time = 0\n",
    "\n",
    "from time import time\n",
    "\n",
    "def onstep(res):\n",
    "    global counter\n",
    "    x0 = res.x_iters   # List of input points\n",
    "    y0 = res.func_vals # Evaluation of input points\n",
    "    print('Last eval: ', x0[-1], \n",
    "          ' - Score ', y0[-1])\n",
    "    print('Current iter: ', counter, \n",
    "          ' - Best Score ', res.fun, \n",
    "          ' - Best Args: ', res.x)\n",
    "    joblib.dump((x0, y0), 'checkpoint.pkl') # Saving a checkpoint to disk\n",
    "    counter += 1\n",
    "    \n",
    "# The objective function to be minimized\n",
    "def make_objective(model_fn, X, space, cv, scoring, validation=0.2, batch_size=50_000):\n",
    "    # This decorator converts your objective function with named arguments into one that\n",
    "    # accepts a list as argument, while doing the conversion automatically.\n",
    "    @use_named_args(space) \n",
    "    def objective(**params):\n",
    "        \n",
    "        print(\"\\nTesting: \", params)\n",
    "        validation_scores = list()\n",
    "        \n",
    "        for k, (train_index, test_index) in enumerate(kf.split(X)):\n",
    "            val_index = list()\n",
    "            train_examples = len(train_index)\n",
    "            train_examples = int(train_examples * (1 - validation))\n",
    "            train_index, val_index = train_index[:train_examples], train_index[train_examples:]\n",
    "            \n",
    "            start_time = time()\n",
    "            \n",
    "            model = model_fn(**params)\n",
    "            measure_to_monitor = 'val_root_mean_squared_error'\n",
    "            modality='min'\n",
    "\n",
    "            early_stopping = tf.keras.callbacks.EarlyStopping(monitor=measure_to_monitor, \n",
    "                                                              mode=modality, \n",
    "                                                              patience=5, \n",
    "                                                              verbose=0)\n",
    "\n",
    "            model_checkpoint = tf.keras.callbacks.ModelCheckpoint('best_model.keras', \n",
    "                                               monitor=measure_to_monitor, \n",
    "                                               mode=modality, \n",
    "                                               save_best_only=True, \n",
    "                                               verbose=0)\n",
    "\n",
    "            run = model.fit(df_to_dataset(X_train.iloc[train_index, :], batch_size=params['batch_size']),\n",
    "                            validation_data=df_to_dataset(X_train.iloc[val_index, :], batch_size=1024),\n",
    "                            epochs=1_000,\n",
    "                            callbacks=[model_checkpoint, early_stopping],\n",
    "                            verbose=0)\n",
    "            \n",
    "            end_time = time()\n",
    "            \n",
    "            rounds = np.argmin(run.history['val_root_mean_squared_error']) + 1\n",
    "            \n",
    "            model = tf.keras.models.load_model('best_model.keras')\n",
    "            os.remove('best_model.keras')\n",
    "            \n",
    "            test_preds = model.predict(df_to_dataset(X.iloc[test_index, :], \n",
    "                                                     shuffle=False,\n",
    "                                                     batch_size=1024), verbose=0).flatten()\n",
    "            test_score = scoring(X.iloc[test_index, :]['target'], test_preds)\n",
    "            print(f\"CV Fold {k+1} rmse:{test_score:0.5f} - {rounds} rounds - it took {end_time-start_time:0.0f} secs\")\n",
    "            validation_scores.append(test_score)\n",
    "            \n",
    "            if len(history[k]) >= 10:\n",
    "                threshold = np.percentile(history[k], q=25)\n",
    "                if test_score > threshold:\n",
    "                    print(f\"Early stopping for under-performing fold: threshold is {threshold:0.5f}\")\n",
    "                    return np.mean(validation_scores)\n",
    "                \n",
    "            history[k].append(test_score)\n",
    "        return np.mean(validation_scores)\n",
    "\n",
    "    return objective"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "a7c7e3c2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-05T23:05:31.085739Z",
     "iopub.status.busy": "2024-10-05T23:05:31.085374Z",
     "iopub.status.idle": "2024-10-05T23:05:31.089729Z",
     "shell.execute_reply": "2024-10-05T23:05:31.088837Z"
    },
    "papermill": {
     "duration": 0.012223,
     "end_time": "2024-10-05T23:05:31.091720",
     "exception": false,
     "start_time": "2024-10-05T23:05:31.079497",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "objective = make_objective(create_model,\n",
    "                           X_train,\n",
    "                           space=space,\n",
    "                           cv=kf,\n",
    "                           scoring=scoring)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "49bb497c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-05T23:05:31.102953Z",
     "iopub.status.busy": "2024-10-05T23:05:31.102658Z",
     "iopub.status.idle": "2024-10-06T00:40:11.457395Z",
     "shell.execute_reply": "2024-10-06T00:40:11.456429Z"
    },
    "papermill": {
     "duration": 5680.364824,
     "end_time": "2024-10-06T00:40:11.461572",
     "exception": false,
     "start_time": "2024-10-05T23:05:31.096748",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 2, 'cat2_dim': 1, 'cat3_dim': 2, 'cat4_dim': 2, 'cat5_dim': 3, 'cat6_dim': 1, 'cat7_dim': 4, 'cat8_dim': 1, 'cat9_dim': 1, 'layers': 5, 'layer_1': 60, 'layer_2': 195, 'layer_3': 232, 'layer_4': 41, 'layer_5': 89, 'activation': <function leaky_relu at 0x79322f30e200>, 'dropout': 0.19639239805041492, 'batch_normalization': False, 'learning_rate': 0.005, 'batch_size': 371}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n",
      "I0000 00:00:1728169537.912735      67 service.cc:145] XLA service 0x7931b40020b0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n",
      "I0000 00:00:1728169537.912799      67 service.cc:153]   StreamExecutor device (0): Tesla P100-PCIE-16GB, Compute Capability 6.0\n",
      "I0000 00:00:1728169547.365487      67 device_compiler.h:188] Compiled cluster using XLA!  This line is logged at most once for the lifetime of the process.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CV Fold 1 rmse:0.74001 - 7 rounds - it took 99 secs\n",
      "CV Fold 2 rmse:0.73718 - 3 rounds - it took 70 secs\n",
      "CV Fold 3 rmse:0.74225 - 6 rounds - it took 89 secs\n",
      "CV Fold 4 rmse:0.73626 - 3 rounds - it took 71 secs\n",
      "CV Fold 5 rmse:0.73594 - 19 rounds - it took 172 secs\n",
      "Last eval:  [2, 2, 1, 2, 2, 3, 1, 4, 1, 1, 5, 60, 195, 232, 41, 89, <function leaky_relu at 0x79322f30e200>, 0.19639239805041492, False, 0.005, 371]  - Score  0.7383292049045582\n",
      "Current iter:  0  - Best Score  0.7383292049045582  - Best Args:  [2, 2, 1, 2, 2, 3, 1, 4, 1, 1, 5, 60, 195, 232, 41, 89, <function leaky_relu at 0x79322f30e200>, 0.19639239805041492, False, 0.005, 371]\n",
      "\n",
      "Testing:  {'cat0_dim': 1, 'cat1_dim': 2, 'cat2_dim': 2, 'cat3_dim': 3, 'cat4_dim': 1, 'cat5_dim': 3, 'cat6_dim': 4, 'cat7_dim': 1, 'cat8_dim': 2, 'cat9_dim': 4, 'layers': 4, 'layer_1': 149, 'layer_2': 144, 'layer_3': 169, 'layer_4': 34, 'layer_5': 195, 'activation': <function leaky_relu at 0x79322f30e200>, 'dropout': 0.47233445852479206, 'batch_normalization': False, 'learning_rate': 0.005, 'batch_size': 931}\n",
      "CV Fold 1 rmse:0.73956 - 26 rounds - it took 216 secs\n",
      "CV Fold 2 rmse:0.73784 - 10 rounds - it took 112 secs\n",
      "CV Fold 3 rmse:0.74606 - 6 rounds - it took 87 secs\n",
      "CV Fold 4 rmse:0.74625 - 4 rounds - it took 74 secs\n",
      "CV Fold 5 rmse:0.73612 - 33 rounds - it took 253 secs\n",
      "Last eval:  [1, 2, 2, 3, 1, 3, 4, 1, 2, 4, 4, 149, 144, 169, 34, 195, <function leaky_relu at 0x79322f30e200>, 0.47233445852479206, False, 0.005, 931]  - Score  0.741167117194071\n",
      "Current iter:  1  - Best Score  0.7383292049045582  - Best Args:  [2, 2, 1, 2, 2, 3, 1, 4, 1, 1, 5, 60, 195, 232, 41, 89, <function leaky_relu at 0x79322f30e200>, 0.19639239805041492, False, 0.005, 371]\n",
      "\n",
      "Testing:  {'cat0_dim': 1, 'cat1_dim': 2, 'cat2_dim': 1, 'cat3_dim': 1, 'cat4_dim': 3, 'cat5_dim': 1, 'cat6_dim': 1, 'cat7_dim': 1, 'cat8_dim': 6, 'cat9_dim': 6, 'layers': 1, 'layer_1': 246, 'layer_2': 19, 'layer_3': 81, 'layer_4': 134, 'layer_5': 107, 'activation': <function leaky_relu at 0x79322f30e200>, 'dropout': 0.30653172894206626, 'batch_normalization': False, 'learning_rate': 0.01, 'batch_size': 313}\n",
      "CV Fold 1 rmse:0.74099 - 2 rounds - it took 53 secs\n",
      "CV Fold 2 rmse:0.73780 - 5 rounds - it took 69 secs\n",
      "CV Fold 3 rmse:0.74372 - 6 rounds - it took 75 secs\n",
      "CV Fold 4 rmse:0.73901 - 1 rounds - it took 45 secs\n",
      "CV Fold 5 rmse:0.73736 - 4 rounds - it took 64 secs\n",
      "Last eval:  [1, 2, 1, 1, 3, 1, 1, 1, 6, 6, 1, 246, 19, 81, 134, 107, <function leaky_relu at 0x79322f30e200>, 0.30653172894206626, False, 0.01, 313]  - Score  0.7397752896020625\n",
      "Current iter:  2  - Best Score  0.7383292049045582  - Best Args:  [2, 2, 1, 2, 2, 3, 1, 4, 1, 1, 5, 60, 195, 232, 41, 89, <function leaky_relu at 0x79322f30e200>, 0.19639239805041492, False, 0.005, 371]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 1, 'cat2_dim': 2, 'cat3_dim': 3, 'cat4_dim': 3, 'cat5_dim': 1, 'cat6_dim': 3, 'cat7_dim': 4, 'cat8_dim': 6, 'cat9_dim': 5, 'layers': 2, 'layer_1': 172, 'layer_2': 86, 'layer_3': 205, 'layer_4': 70, 'layer_5': 8, 'activation': <function leaky_relu at 0x79322f30e200>, 'dropout': 0.10443837804741737, 'batch_normalization': True, 'learning_rate': 0.002, 'batch_size': 399}\n",
      "CV Fold 1 rmse:0.73851 - 23 rounds - it took 191 secs\n",
      "CV Fold 2 rmse:0.73656 - 6 rounds - it took 81 secs\n",
      "CV Fold 3 rmse:0.74176 - 16 rounds - it took 142 secs\n",
      "CV Fold 4 rmse:0.73562 - 6 rounds - it took 81 secs\n",
      "CV Fold 5 rmse:0.73563 - 15 rounds - it took 137 secs\n",
      "Last eval:  [2, 1, 2, 3, 3, 1, 3, 4, 6, 5, 2, 172, 86, 205, 70, 8, <function leaky_relu at 0x79322f30e200>, 0.10443837804741737, True, 0.002, 399]  - Score  0.737617355775065\n",
      "Current iter:  3  - Best Score  0.737617355775065  - Best Args:  [2, 1, 2, 3, 3, 1, 3, 4, 6, 5, 2, 172, 86, 205, 70, 8, <function leaky_relu at 0x79322f30e200>, 0.10443837804741737, True, 0.002, 399]\n",
      "\n",
      "Testing:  {'cat0_dim': 1, 'cat1_dim': 2, 'cat2_dim': 1, 'cat3_dim': 2, 'cat4_dim': 2, 'cat5_dim': 2, 'cat6_dim': 1, 'cat7_dim': 2, 'cat8_dim': 6, 'cat9_dim': 2, 'layers': 4, 'layer_1': 194, 'layer_2': 97, 'layer_3': 199, 'layer_4': 96, 'layer_5': 2, 'activation': <function relu at 0x79322f471900>, 'dropout': 0.33682981541789475, 'batch_normalization': False, 'learning_rate': 0.001, 'batch_size': 810}\n",
      "CV Fold 1 rmse:0.74018 - 14 rounds - it took 138 secs\n",
      "CV Fold 2 rmse:0.73720 - 17 rounds - it took 158 secs\n",
      "CV Fold 3 rmse:0.74498 - 9 rounds - it took 105 secs\n",
      "CV Fold 4 rmse:0.73657 - 12 rounds - it took 123 secs\n",
      "CV Fold 5 rmse:0.73977 - 6 rounds - it took 86 secs\n",
      "Last eval:  [1, 2, 1, 2, 2, 2, 1, 2, 6, 2, 4, 194, 97, 199, 96, 2, <function relu at 0x79322f471900>, 0.33682981541789475, False, 0.001, 810]  - Score  0.7397393533668063\n",
      "Current iter:  4  - Best Score  0.737617355775065  - Best Args:  [2, 1, 2, 3, 3, 1, 3, 4, 6, 5, 2, 172, 86, 205, 70, 8, <function leaky_relu at 0x79322f30e200>, 0.10443837804741737, True, 0.002, 399]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 1, 'cat2_dim': 2, 'cat3_dim': 2, 'cat4_dim': 2, 'cat5_dim': 1, 'cat6_dim': 3, 'cat7_dim': 3, 'cat8_dim': 4, 'cat9_dim': 3, 'layers': 4, 'layer_1': 132, 'layer_2': 123, 'layer_3': 100, 'layer_4': 64, 'layer_5': 165, 'activation': <function relu at 0x79322f471900>, 'dropout': 0.10841106881377147, 'batch_normalization': False, 'learning_rate': 0.001, 'batch_size': 818}\n",
      "CV Fold 1 rmse:0.73924 - 22 rounds - it took 189 secs\n",
      "CV Fold 2 rmse:0.73625 - 17 rounds - it took 155 secs\n",
      "CV Fold 3 rmse:0.76286 - 14 rounds - it took 136 secs\n",
      "CV Fold 4 rmse:0.73578 - 18 rounds - it took 159 secs\n",
      "CV Fold 5 rmse:0.73655 - 11 rounds - it took 115 secs\n",
      "Last eval:  [2, 1, 2, 2, 2, 1, 3, 3, 4, 3, 4, 132, 123, 100, 64, 165, <function relu at 0x79322f471900>, 0.10841106881377147, False, 0.001, 818]  - Score  0.7421347512762824\n",
      "Current iter:  5  - Best Score  0.737617355775065  - Best Args:  [2, 1, 2, 3, 3, 1, 3, 4, 6, 5, 2, 172, 86, 205, 70, 8, <function leaky_relu at 0x79322f30e200>, 0.10443837804741737, True, 0.002, 399]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 1, 'cat2_dim': 2, 'cat3_dim': 3, 'cat4_dim': 2, 'cat5_dim': 1, 'cat6_dim': 3, 'cat7_dim': 3, 'cat8_dim': 4, 'cat9_dim': 1, 'layers': 4, 'layer_1': 4, 'layer_2': 53, 'layer_3': 82, 'layer_4': 34, 'layer_5': 184, 'activation': <function leaky_relu at 0x79322f30e200>, 'dropout': 0.09159568100355843, 'batch_normalization': False, 'learning_rate': 0.01, 'batch_size': 744}\n",
      "CV Fold 1 rmse:0.74030 - 10 rounds - it took 113 secs\n",
      "CV Fold 2 rmse:0.73682 - 12 rounds - it took 123 secs\n",
      "CV Fold 3 rmse:0.74372 - 8 rounds - it took 99 secs\n",
      "CV Fold 4 rmse:0.73641 - 11 rounds - it took 116 secs\n",
      "CV Fold 5 rmse:0.73681 - 12 rounds - it took 123 secs\n",
      "Last eval:  [2, 1, 2, 3, 2, 1, 3, 3, 4, 1, 4, 4, 53, 82, 34, 184, <function leaky_relu at 0x79322f30e200>, 0.09159568100355843, False, 0.01, 744]  - Score  0.7388120755937619\n",
      "Current iter:  6  - Best Score  0.737617355775065  - Best Args:  [2, 1, 2, 3, 3, 1, 3, 4, 6, 5, 2, 172, 86, 205, 70, 8, <function leaky_relu at 0x79322f30e200>, 0.10443837804741737, True, 0.002, 399]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 1, 'cat2_dim': 1, 'cat3_dim': 2, 'cat4_dim': 3, 'cat5_dim': 3, 'cat6_dim': 3, 'cat7_dim': 4, 'cat8_dim': 6, 'cat9_dim': 1, 'layers': 1, 'layer_1': 74, 'layer_2': 21, 'layer_3': 97, 'layer_4': 74, 'layer_5': 156, 'activation': <function relu at 0x79322f471900>, 'dropout': 0.11154081632030918, 'batch_normalization': False, 'learning_rate': 0.005, 'batch_size': 516}\n",
      "CV Fold 1 rmse:0.74081 - 3 rounds - it took 58 secs\n",
      "CV Fold 2 rmse:0.73721 - 8 rounds - it took 86 secs\n",
      "CV Fold 3 rmse:0.74388 - 2 rounds - it took 50 secs\n",
      "CV Fold 4 rmse:0.73626 - 7 rounds - it took 82 secs\n",
      "CV Fold 5 rmse:0.73663 - 16 rounds - it took 138 secs\n",
      "Last eval:  [2, 1, 1, 2, 3, 3, 3, 4, 6, 1, 1, 74, 21, 97, 74, 156, <function relu at 0x79322f471900>, 0.11154081632030918, False, 0.005, 516]  - Score  0.738959482788063\n",
      "Current iter:  7  - Best Score  0.737617355775065  - Best Args:  [2, 1, 2, 3, 3, 1, 3, 4, 6, 5, 2, 172, 86, 205, 70, 8, <function leaky_relu at 0x79322f30e200>, 0.10443837804741737, True, 0.002, 399]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 2, 'cat2_dim': 2, 'cat3_dim': 1, 'cat4_dim': 3, 'cat5_dim': 2, 'cat6_dim': 2, 'cat7_dim': 2, 'cat8_dim': 4, 'cat9_dim': 1, 'layers': 1, 'layer_1': 154, 'layer_2': 159, 'layer_3': 151, 'layer_4': 112, 'layer_5': 27, 'activation': <function relu at 0x79322f471900>, 'dropout': 0.3298342060347065, 'batch_normalization': False, 'learning_rate': 0.01, 'batch_size': 484}\n",
      "CV Fold 1 rmse:0.74070 - 5 rounds - it took 70 secs\n",
      "CV Fold 2 rmse:0.73836 - 3 rounds - it took 56 secs\n",
      "CV Fold 3 rmse:0.74313 - 9 rounds - it took 93 secs\n",
      "CV Fold 4 rmse:0.73623 - 7 rounds - it took 82 secs\n",
      "CV Fold 5 rmse:0.73721 - 4 rounds - it took 62 secs\n",
      "Last eval:  [2, 2, 2, 1, 3, 2, 2, 2, 4, 1, 1, 154, 159, 151, 112, 27, <function relu at 0x79322f471900>, 0.3298342060347065, False, 0.01, 484]  - Score  0.7391244420072409\n",
      "Current iter:  8  - Best Score  0.737617355775065  - Best Args:  [2, 1, 2, 3, 3, 1, 3, 4, 6, 5, 2, 172, 86, 205, 70, 8, <function leaky_relu at 0x79322f30e200>, 0.10443837804741737, True, 0.002, 399]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 2, 'cat2_dim': 2, 'cat3_dim': 3, 'cat4_dim': 2, 'cat5_dim': 2, 'cat6_dim': 4, 'cat7_dim': 1, 'cat8_dim': 1, 'cat9_dim': 8, 'layers': 2, 'layer_1': 20, 'layer_2': 42, 'layer_3': 184, 'layer_4': 209, 'layer_5': 13, 'activation': <function leaky_relu at 0x79322f30e200>, 'dropout': 0.17702333104190512, 'batch_normalization': False, 'learning_rate': 0.01, 'batch_size': 472}\n",
      "CV Fold 1 rmse:0.73998 - 10 rounds - it took 105 secs\n",
      "CV Fold 2 rmse:0.73619 - 23 rounds - it took 183 secs\n",
      "CV Fold 3 rmse:0.74271 - 26 rounds - it took 201 secs\n",
      "CV Fold 4 rmse:0.73554 - 13 rounds - it took 120 secs\n",
      "CV Fold 5 rmse:0.73743 - 4 rounds - it took 67 secs\n",
      "Last eval:  [2, 2, 2, 3, 2, 2, 4, 1, 1, 8, 2, 20, 42, 184, 209, 13, <function leaky_relu at 0x79322f30e200>, 0.17702333104190512, False, 0.01, 472]  - Score  0.7383688549900136\n",
      "Current iter:  9  - Best Score  0.737617355775065  - Best Args:  [2, 1, 2, 3, 3, 1, 3, 4, 6, 5, 2, 172, 86, 205, 70, 8, <function leaky_relu at 0x79322f30e200>, 0.10443837804741737, True, 0.002, 399]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "84734"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gp_round = dummy_minimize(func=objective,\n",
    "                          dimensions=space,\n",
    "                          n_calls=10,\n",
    "                          callback=[onstep],\n",
    "                          random_state=0)\n",
    "\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "00f880b4",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-06T00:40:11.494816Z",
     "iopub.status.busy": "2024-10-06T00:40:11.494193Z",
     "iopub.status.idle": "2024-10-06T00:40:11.502177Z",
     "shell.execute_reply": "2024-10-06T00:40:11.501185Z"
    },
    "papermill": {
     "duration": 0.033114,
     "end_time": "2024-10-06T00:40:11.504160",
     "exception": false,
     "start_time": "2024-10-06T00:40:11.471046",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10\n"
     ]
    }
   ],
   "source": [
    "x0, y0 = joblib.load('checkpoint.pkl')\n",
    "print(len(x0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "96cc9002",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-06T00:40:11.525963Z",
     "iopub.status.busy": "2024-10-06T00:40:11.525225Z",
     "iopub.status.idle": "2024-10-06T01:55:34.895324Z",
     "shell.execute_reply": "2024-10-06T01:55:34.894393Z"
    },
    "papermill": {
     "duration": 4523.38271,
     "end_time": "2024-10-06T01:55:34.897512",
     "exception": false,
     "start_time": "2024-10-06T00:40:11.514802",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Last eval:  [2, 2, 2, 3, 2, 2, 4, 1, 1, 8, 2, 20, 42, 184, 209, 13, <function leaky_relu at 0x79322f30e200>, 0.17702333104190512, False, 0.01, 472]  - Score  0.7383688549900136\n",
      "Current iter:  10  - Best Score  0.737617355775065  - Best Args:  [2, 1, 2, 3, 3, 1, 3, 4, 6, 5, 2, 172, 86, 205, 70, 8, <function leaky_relu at 0x79322f30e200>, 0.10443837804741737, True, 0.002, 399]\n",
      "\n",
      "Testing:  {'cat0_dim': 1, 'cat1_dim': 1, 'cat2_dim': 2, 'cat3_dim': 1, 'cat4_dim': 1, 'cat5_dim': 3, 'cat6_dim': 2, 'cat7_dim': 4, 'cat8_dim': 2, 'cat9_dim': 2, 'layers': 2, 'layer_1': 79, 'layer_2': 228, 'layer_3': 3, 'layer_4': 55, 'layer_5': 204, 'activation': <function leaky_relu at 0x79322f30e200>, 'dropout': 0.105496665074676, 'batch_normalization': True, 'learning_rate': 0.001, 'batch_size': 956}\n",
      "CV Fold 1 rmse:0.73896 - 37 rounds - it took 281 secs\n",
      "CV Fold 2 rmse:0.74849 - 1 rounds - it took 50 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73663\n",
      "Last eval:  [1, 1, 2, 1, 1, 3, 2, 4, 2, 2, 2, 79, 228, 3, 55, 204, <function leaky_relu at 0x79322f30e200>, 0.105496665074676, True, 0.001, 956]  - Score  0.7437289695717104\n",
      "Current iter:  11  - Best Score  0.737617355775065  - Best Args:  [2, 1, 2, 3, 3, 1, 3, 4, 6, 5, 2, 172, 86, 205, 70, 8, <function leaky_relu at 0x79322f30e200>, 0.10443837804741737, True, 0.002, 399]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 2, 'cat2_dim': 1, 'cat3_dim': 2, 'cat4_dim': 2, 'cat5_dim': 1, 'cat6_dim': 1, 'cat7_dim': 2, 'cat8_dim': 2, 'cat9_dim': 8, 'layers': 4, 'layer_1': 226, 'layer_2': 256, 'layer_3': 2, 'layer_4': 158, 'layer_5': 2, 'activation': <function leaky_relu at 0x79322f30e200>, 'dropout': 0.0, 'batch_normalization': False, 'learning_rate': 0.002, 'batch_size': 256}\n",
      "CV Fold 1 rmse:0.74165 - 5 rounds - it took 70 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73940\n",
      "Last eval:  [2, 2, 1, 2, 2, 1, 1, 2, 2, 8, 4, 226, 256, 2, 158, 2, <function leaky_relu at 0x79322f30e200>, 0.0, False, 0.002, 256]  - Score  0.7416525234255783\n",
      "Current iter:  12  - Best Score  0.737617355775065  - Best Args:  [2, 1, 2, 3, 3, 1, 3, 4, 6, 5, 2, 172, 86, 205, 70, 8, <function leaky_relu at 0x79322f30e200>, 0.10443837804741737, True, 0.002, 399]\n",
      "\n",
      "Testing:  {'cat0_dim': 1, 'cat1_dim': 2, 'cat2_dim': 1, 'cat3_dim': 2, 'cat4_dim': 2, 'cat5_dim': 2, 'cat6_dim': 3, 'cat7_dim': 2, 'cat8_dim': 4, 'cat9_dim': 3, 'layers': 3, 'layer_1': 124, 'layer_2': 17, 'layer_3': 227, 'layer_4': 188, 'layer_5': 42, 'activation': <function relu at 0x79322f471900>, 'dropout': 0.49680294063576314, 'batch_normalization': True, 'learning_rate': 0.005, 'batch_size': 943}\n",
      "CV Fold 1 rmse:0.74054 - 16 rounds - it took 152 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73940\n",
      "Last eval:  [1, 2, 1, 2, 2, 2, 3, 2, 4, 3, 3, 124, 17, 227, 188, 42, <function relu at 0x79322f471900>, 0.49680294063576314, True, 0.005, 943]  - Score  0.7405429077532458\n",
      "Current iter:  13  - Best Score  0.737617355775065  - Best Args:  [2, 1, 2, 3, 3, 1, 3, 4, 6, 5, 2, 172, 86, 205, 70, 8, <function leaky_relu at 0x79322f30e200>, 0.10443837804741737, True, 0.002, 399]\n",
      "\n",
      "Testing:  {'cat0_dim': 1, 'cat1_dim': 1, 'cat2_dim': 2, 'cat3_dim': 2, 'cat4_dim': 2, 'cat5_dim': 1, 'cat6_dim': 3, 'cat7_dim': 1, 'cat8_dim': 3, 'cat9_dim': 3, 'layers': 5, 'layer_1': 34, 'layer_2': 74, 'layer_3': 11, 'layer_4': 231, 'layer_5': 89, 'activation': <function relu at 0x79322f471900>, 'dropout': 0.4980710112638439, 'batch_normalization': True, 'learning_rate': 0.001, 'batch_size': 327}\n",
      "CV Fold 1 rmse:0.74035 - 23 rounds - it took 206 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73940\n",
      "Last eval:  [1, 1, 2, 2, 2, 1, 3, 1, 3, 3, 5, 34, 74, 11, 231, 89, <function relu at 0x79322f471900>, 0.4980710112638439, True, 0.001, 327]  - Score  0.7403511620121912\n",
      "Current iter:  14  - Best Score  0.737617355775065  - Best Args:  [2, 1, 2, 3, 3, 1, 3, 4, 6, 5, 2, 172, 86, 205, 70, 8, <function leaky_relu at 0x79322f30e200>, 0.10443837804741737, True, 0.002, 399]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 2, 'cat2_dim': 1, 'cat3_dim': 2, 'cat4_dim': 3, 'cat5_dim': 3, 'cat6_dim': 2, 'cat7_dim': 2, 'cat8_dim': 3, 'cat9_dim': 2, 'layers': 5, 'layer_1': 219, 'layer_2': 89, 'layer_3': 246, 'layer_4': 175, 'layer_5': 79, 'activation': <function relu at 0x79322f471900>, 'dropout': 0.04619831589851348, 'batch_normalization': True, 'learning_rate': 0.001, 'batch_size': 265}\n",
      "CV Fold 1 rmse:0.73805 - 21 rounds - it took 202 secs\n",
      "CV Fold 2 rmse:0.73559 - 12 rounds - it took 140 secs\n",
      "CV Fold 3 rmse:0.74069 - 19 rounds - it took 186 secs\n",
      "CV Fold 4 rmse:0.73511 - 12 rounds - it took 139 secs\n",
      "CV Fold 5 rmse:0.73543 - 21 rounds - it took 201 secs\n",
      "Last eval:  [2, 2, 1, 2, 3, 3, 2, 2, 3, 2, 5, 219, 89, 246, 175, 79, <function relu at 0x79322f471900>, 0.04619831589851348, True, 0.001, 265]  - Score  0.7369712456359043\n",
      "Current iter:  15  - Best Score  0.7369712456359043  - Best Args:  [2, 2, 1, 2, 3, 3, 2, 2, 3, 2, 5, 219, 89, 246, 175, 79, <function relu at 0x79322f471900>, 0.04619831589851348, True, 0.001, 265]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 2, 'cat2_dim': 1, 'cat3_dim': 2, 'cat4_dim': 2, 'cat5_dim': 2, 'cat6_dim': 3, 'cat7_dim': 2, 'cat8_dim': 5, 'cat9_dim': 7, 'layers': 3, 'layer_1': 167, 'layer_2': 223, 'layer_3': 240, 'layer_4': 161, 'layer_5': 200, 'activation': <function leaky_relu at 0x79322f30e200>, 'dropout': 0.40085245048990814, 'batch_normalization': False, 'learning_rate': 0.01, 'batch_size': 1023}\n",
      "CV Fold 1 rmse:0.74013 - 9 rounds - it took 103 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73917\n",
      "Last eval:  [2, 2, 1, 2, 2, 2, 3, 2, 5, 7, 3, 167, 223, 240, 161, 200, <function leaky_relu at 0x79322f30e200>, 0.40085245048990814, False, 0.01, 1023]  - Score  0.7401285546986865\n",
      "Current iter:  16  - Best Score  0.7369712456359043  - Best Args:  [2, 2, 1, 2, 3, 3, 2, 2, 3, 2, 5, 219, 89, 246, 175, 79, <function relu at 0x79322f471900>, 0.04619831589851348, True, 0.001, 265]\n",
      "\n",
      "Testing:  {'cat0_dim': 1, 'cat1_dim': 2, 'cat2_dim': 1, 'cat3_dim': 1, 'cat4_dim': 1, 'cat5_dim': 1, 'cat6_dim': 4, 'cat7_dim': 2, 'cat8_dim': 6, 'cat9_dim': 1, 'layers': 1, 'layer_1': 115, 'layer_2': 9, 'layer_3': 3, 'layer_4': 159, 'layer_5': 249, 'activation': <function leaky_relu at 0x79322f30e200>, 'dropout': 0.07762592760542704, 'batch_normalization': False, 'learning_rate': 0.01, 'batch_size': 1017}\n",
      "CV Fold 1 rmse:0.74078 - 3 rounds - it took 59 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73917\n",
      "Last eval:  [1, 2, 1, 1, 1, 1, 4, 2, 6, 1, 1, 115, 9, 3, 159, 249, <function leaky_relu at 0x79322f30e200>, 0.07762592760542704, False, 0.01, 1017]  - Score  0.7407799578504294\n",
      "Current iter:  17  - Best Score  0.7369712456359043  - Best Args:  [2, 2, 1, 2, 3, 3, 2, 2, 3, 2, 5, 219, 89, 246, 175, 79, <function relu at 0x79322f471900>, 0.04619831589851348, True, 0.001, 265]\n",
      "\n",
      "Testing:  {'cat0_dim': 1, 'cat1_dim': 2, 'cat2_dim': 2, 'cat3_dim': 2, 'cat4_dim': 2, 'cat5_dim': 1, 'cat6_dim': 1, 'cat7_dim': 2, 'cat8_dim': 1, 'cat9_dim': 5, 'layers': 2, 'layer_1': 191, 'layer_2': 255, 'layer_3': 253, 'layer_4': 38, 'layer_5': 55, 'activation': <function leaky_relu at 0x79322f30e200>, 'dropout': 0.006791295089060569, 'batch_normalization': False, 'learning_rate': 0.01, 'batch_size': 262}\n",
      "CV Fold 1 rmse:0.74118 - 7 rounds - it took 88 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73917\n",
      "Last eval:  [1, 2, 2, 2, 2, 1, 1, 2, 1, 5, 2, 191, 255, 253, 38, 55, <function leaky_relu at 0x79322f30e200>, 0.006791295089060569, False, 0.01, 262]  - Score  0.7411753875769175\n",
      "Current iter:  18  - Best Score  0.7369712456359043  - Best Args:  [2, 2, 1, 2, 3, 3, 2, 2, 3, 2, 5, 219, 89, 246, 175, 79, <function relu at 0x79322f471900>, 0.04619831589851348, True, 0.001, 265]\n",
      "\n",
      "Testing:  {'cat0_dim': 1, 'cat1_dim': 1, 'cat2_dim': 2, 'cat3_dim': 2, 'cat4_dim': 1, 'cat5_dim': 2, 'cat6_dim': 4, 'cat7_dim': 1, 'cat8_dim': 3, 'cat9_dim': 2, 'layers': 2, 'layer_1': 125, 'layer_2': 158, 'layer_3': 252, 'layer_4': 107, 'layer_5': 117, 'activation': <function leaky_relu at 0x79322f30e200>, 'dropout': 0.49205542030061206, 'batch_normalization': False, 'learning_rate': 0.001, 'batch_size': 278}\n",
      "CV Fold 1 rmse:0.74064 - 7 rounds - it took 89 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73917\n",
      "Last eval:  [1, 1, 2, 2, 1, 2, 4, 1, 3, 2, 2, 125, 158, 252, 107, 117, <function leaky_relu at 0x79322f30e200>, 0.49205542030061206, False, 0.001, 278]  - Score  0.7406433195050173\n",
      "Current iter:  19  - Best Score  0.7369712456359043  - Best Args:  [2, 2, 1, 2, 3, 3, 2, 2, 3, 2, 5, 219, 89, 246, 175, 79, <function relu at 0x79322f471900>, 0.04619831589851348, True, 0.001, 265]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 2, 'cat2_dim': 2, 'cat3_dim': 2, 'cat4_dim': 2, 'cat5_dim': 1, 'cat6_dim': 4, 'cat7_dim': 2, 'cat8_dim': 5, 'cat9_dim': 2, 'layers': 3, 'layer_1': 45, 'layer_2': 30, 'layer_3': 240, 'layer_4': 23, 'layer_5': 195, 'activation': <function leaky_relu at 0x79322f30e200>, 'dropout': 0.18456154247676712, 'batch_normalization': False, 'learning_rate': 0.001, 'batch_size': 993}\n",
      "CV Fold 1 rmse:0.73951 - 49 rounds - it took 354 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73917\n",
      "Last eval:  [2, 2, 2, 2, 2, 1, 4, 2, 5, 2, 3, 45, 30, 240, 23, 195, <function leaky_relu at 0x79322f30e200>, 0.18456154247676712, False, 0.001, 993]  - Score  0.7395111036004236\n",
      "Current iter:  20  - Best Score  0.7369712456359043  - Best Args:  [2, 2, 1, 2, 3, 3, 2, 2, 3, 2, 5, 219, 89, 246, 175, 79, <function relu at 0x79322f471900>, 0.04619831589851348, True, 0.001, 265]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 2, 'cat2_dim': 2, 'cat3_dim': 2, 'cat4_dim': 2, 'cat5_dim': 2, 'cat6_dim': 3, 'cat7_dim': 2, 'cat8_dim': 5, 'cat9_dim': 4, 'layers': 3, 'layer_1': 149, 'layer_2': 2, 'layer_3': 256, 'layer_4': 226, 'layer_5': 141, 'activation': <function relu at 0x79322f471900>, 'dropout': 0.38284274618453945, 'batch_normalization': True, 'learning_rate': 0.001, 'batch_size': 256}\n",
      "CV Fold 1 rmse:0.74119 - 10 rounds - it took 109 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73917\n",
      "Last eval:  [2, 2, 2, 2, 2, 2, 3, 2, 5, 4, 3, 149, 2, 256, 226, 141, <function relu at 0x79322f471900>, 0.38284274618453945, True, 0.001, 256]  - Score  0.7411891446895763\n",
      "Current iter:  21  - Best Score  0.7369712456359043  - Best Args:  [2, 2, 1, 2, 3, 3, 2, 2, 3, 2, 5, 219, 89, 246, 175, 79, <function relu at 0x79322f471900>, 0.04619831589851348, True, 0.001, 265]\n",
      "\n",
      "Testing:  {'cat0_dim': 1, 'cat1_dim': 1, 'cat2_dim': 2, 'cat3_dim': 2, 'cat4_dim': 3, 'cat5_dim': 3, 'cat6_dim': 4, 'cat7_dim': 2, 'cat8_dim': 1, 'cat9_dim': 3, 'layers': 4, 'layer_1': 2, 'layer_2': 235, 'layer_3': 211, 'layer_4': 31, 'layer_5': 121, 'activation': <function leaky_relu at 0x79322f30e200>, 'dropout': 0.06707363300705028, 'batch_normalization': False, 'learning_rate': 0.002, 'batch_size': 558}\n",
      "CV Fold 1 rmse:0.74171 - 16 rounds - it took 148 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73917\n",
      "Last eval:  [1, 1, 2, 2, 3, 3, 4, 2, 1, 3, 4, 2, 235, 211, 31, 121, <function leaky_relu at 0x79322f30e200>, 0.06707363300705028, False, 0.002, 558]  - Score  0.7417124212592358\n",
      "Current iter:  22  - Best Score  0.7369712456359043  - Best Args:  [2, 2, 1, 2, 3, 3, 2, 2, 3, 2, 5, 219, 89, 246, 175, 79, <function relu at 0x79322f471900>, 0.04619831589851348, True, 0.001, 265]\n",
      "\n",
      "Testing:  {'cat0_dim': 1, 'cat1_dim': 2, 'cat2_dim': 1, 'cat3_dim': 1, 'cat4_dim': 1, 'cat5_dim': 3, 'cat6_dim': 2, 'cat7_dim': 4, 'cat8_dim': 3, 'cat9_dim': 5, 'layers': 5, 'layer_1': 118, 'layer_2': 16, 'layer_3': 190, 'layer_4': 158, 'layer_5': 110, 'activation': <function leaky_relu at 0x79322f30e200>, 'dropout': 0.03716056577749933, 'batch_normalization': False, 'learning_rate': 0.005, 'batch_size': 747}\n",
      "CV Fold 1 rmse:0.74365 - 1 rounds - it took 65 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73917\n",
      "Last eval:  [1, 2, 1, 1, 1, 3, 2, 4, 3, 5, 5, 118, 16, 190, 158, 110, <function leaky_relu at 0x79322f30e200>, 0.03716056577749933, False, 0.005, 747]  - Score  0.7436479811147652\n",
      "Current iter:  23  - Best Score  0.7369712456359043  - Best Args:  [2, 2, 1, 2, 3, 3, 2, 2, 3, 2, 5, 219, 89, 246, 175, 79, <function relu at 0x79322f471900>, 0.04619831589851348, True, 0.001, 265]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 2, 'cat2_dim': 2, 'cat3_dim': 1, 'cat4_dim': 1, 'cat5_dim': 2, 'cat6_dim': 3, 'cat7_dim': 1, 'cat8_dim': 1, 'cat9_dim': 5, 'layers': 4, 'layer_1': 104, 'layer_2': 8, 'layer_3': 173, 'layer_4': 200, 'layer_5': 198, 'activation': <function leaky_relu at 0x79322f30e200>, 'dropout': 0.06692398612508772, 'batch_normalization': False, 'learning_rate': 0.005, 'batch_size': 563}\n",
      "CV Fold 1 rmse:0.74020 - 3 rounds - it took 70 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73917\n",
      "Last eval:  [2, 2, 2, 1, 1, 2, 3, 1, 1, 5, 4, 104, 8, 173, 200, 198, <function leaky_relu at 0x79322f30e200>, 0.06692398612508772, False, 0.005, 563]  - Score  0.7401994120078188\n",
      "Current iter:  24  - Best Score  0.7369712456359043  - Best Args:  [2, 2, 1, 2, 3, 3, 2, 2, 3, 2, 5, 219, 89, 246, 175, 79, <function relu at 0x79322f471900>, 0.04619831589851348, True, 0.001, 265]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 1, 'cat2_dim': 1, 'cat3_dim': 2, 'cat4_dim': 1, 'cat5_dim': 2, 'cat6_dim': 2, 'cat7_dim': 1, 'cat8_dim': 3, 'cat9_dim': 6, 'layers': 3, 'layer_1': 255, 'layer_2': 250, 'layer_3': 44, 'layer_4': 107, 'layer_5': 36, 'activation': <function relu at 0x79322f471900>, 'dropout': 0.4868761996121825, 'batch_normalization': False, 'learning_rate': 0.01, 'batch_size': 469}\n",
      "CV Fold 1 rmse:0.74018 - 12 rounds - it took 123 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73917\n",
      "Last eval:  [2, 1, 1, 2, 1, 2, 2, 1, 3, 6, 3, 255, 250, 44, 107, 36, <function relu at 0x79322f471900>, 0.4868761996121825, False, 0.01, 469]  - Score  0.7401798065754588\n",
      "Current iter:  25  - Best Score  0.7369712456359043  - Best Args:  [2, 2, 1, 2, 3, 3, 2, 2, 3, 2, 5, 219, 89, 246, 175, 79, <function relu at 0x79322f471900>, 0.04619831589851348, True, 0.001, 265]\n",
      "\n",
      "Testing:  {'cat0_dim': 1, 'cat1_dim': 2, 'cat2_dim': 2, 'cat3_dim': 1, 'cat4_dim': 3, 'cat5_dim': 1, 'cat6_dim': 2, 'cat7_dim': 2, 'cat8_dim': 3, 'cat9_dim': 1, 'layers': 3, 'layer_1': 5, 'layer_2': 95, 'layer_3': 138, 'layer_4': 30, 'layer_5': 134, 'activation': <function relu at 0x79322f471900>, 'dropout': 0.4925616680424701, 'batch_normalization': False, 'learning_rate': 0.001, 'batch_size': 748}\n",
      "CV Fold 1 rmse:0.74822 - 14 rounds - it took 138 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73917\n",
      "Last eval:  [1, 2, 2, 1, 3, 1, 2, 2, 3, 1, 3, 5, 95, 138, 30, 134, <function relu at 0x79322f471900>, 0.4925616680424701, False, 0.001, 748]  - Score  0.7482192226855737\n",
      "Current iter:  26  - Best Score  0.7369712456359043  - Best Args:  [2, 2, 1, 2, 3, 3, 2, 2, 3, 2, 5, 219, 89, 246, 175, 79, <function relu at 0x79322f471900>, 0.04619831589851348, True, 0.001, 265]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 2, 'cat2_dim': 1, 'cat3_dim': 3, 'cat4_dim': 1, 'cat5_dim': 1, 'cat6_dim': 1, 'cat7_dim': 4, 'cat8_dim': 5, 'cat9_dim': 8, 'layers': 3, 'layer_1': 164, 'layer_2': 193, 'layer_3': 244, 'layer_4': 88, 'layer_5': 169, 'activation': <function relu at 0x79322f471900>, 'dropout': 0.273996259572315, 'batch_normalization': False, 'learning_rate': 0.01, 'batch_size': 316}\n",
      "CV Fold 1 rmse:0.74010 - 5 rounds - it took 77 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73917\n",
      "Last eval:  [2, 2, 1, 3, 1, 1, 1, 4, 5, 8, 3, 164, 193, 244, 88, 169, <function relu at 0x79322f471900>, 0.273996259572315, False, 0.01, 316]  - Score  0.7400990953230268\n",
      "Current iter:  27  - Best Score  0.7369712456359043  - Best Args:  [2, 2, 1, 2, 3, 3, 2, 2, 3, 2, 5, 219, 89, 246, 175, 79, <function relu at 0x79322f471900>, 0.04619831589851348, True, 0.001, 265]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 2, 'cat2_dim': 2, 'cat3_dim': 3, 'cat4_dim': 3, 'cat5_dim': 3, 'cat6_dim': 3, 'cat7_dim': 2, 'cat8_dim': 2, 'cat9_dim': 2, 'layers': 2, 'layer_1': 140, 'layer_2': 144, 'layer_3': 256, 'layer_4': 2, 'layer_5': 124, 'activation': <function leaky_relu at 0x79322f30e200>, 'dropout': 0.18003190500811303, 'batch_normalization': True, 'learning_rate': 0.005, 'batch_size': 981}\n",
      "CV Fold 1 rmse:0.73966 - 5 rounds - it took 76 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73917\n",
      "Last eval:  [2, 2, 2, 3, 3, 3, 3, 2, 2, 2, 2, 140, 144, 256, 2, 124, <function leaky_relu at 0x79322f30e200>, 0.18003190500811303, True, 0.005, 981]  - Score  0.739662701164303\n",
      "Current iter:  28  - Best Score  0.7369712456359043  - Best Args:  [2, 2, 1, 2, 3, 3, 2, 2, 3, 2, 5, 219, 89, 246, 175, 79, <function relu at 0x79322f471900>, 0.04619831589851348, True, 0.001, 265]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 2, 'cat2_dim': 1, 'cat3_dim': 1, 'cat4_dim': 3, 'cat5_dim': 3, 'cat6_dim': 4, 'cat7_dim': 4, 'cat8_dim': 1, 'cat9_dim': 7, 'layers': 2, 'layer_1': 66, 'layer_2': 126, 'layer_3': 3, 'layer_4': 7, 'layer_5': 50, 'activation': <function relu at 0x79322f471900>, 'dropout': 0.4349077844485195, 'batch_normalization': False, 'learning_rate': 0.01, 'batch_size': 315}\n",
      "CV Fold 1 rmse:0.74004 - 7 rounds - it took 87 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73917\n",
      "Last eval:  [2, 2, 1, 1, 3, 3, 4, 4, 1, 7, 2, 66, 126, 3, 7, 50, <function relu at 0x79322f471900>, 0.4349077844485195, False, 0.01, 315]  - Score  0.7400426939511701\n",
      "Current iter:  29  - Best Score  0.7369712456359043  - Best Args:  [2, 2, 1, 2, 3, 3, 2, 2, 3, 2, 5, 219, 89, 246, 175, 79, <function relu at 0x79322f471900>, 0.04619831589851348, True, 0.001, 265]\n",
      "\n",
      "Testing:  {'cat0_dim': 1, 'cat1_dim': 2, 'cat2_dim': 1, 'cat3_dim': 3, 'cat4_dim': 2, 'cat5_dim': 1, 'cat6_dim': 3, 'cat7_dim': 2, 'cat8_dim': 3, 'cat9_dim': 7, 'layers': 5, 'layer_1': 191, 'layer_2': 24, 'layer_3': 13, 'layer_4': 135, 'layer_5': 148, 'activation': <function relu at 0x79322f471900>, 'dropout': 0.1870569907990092, 'batch_normalization': True, 'learning_rate': 0.01, 'batch_size': 279}\n",
      "CV Fold 1 rmse:0.74195 - 6 rounds - it took 99 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73917\n",
      "Last eval:  [1, 2, 1, 3, 2, 1, 3, 2, 3, 7, 5, 191, 24, 13, 135, 148, <function relu at 0x79322f471900>, 0.1870569907990092, True, 0.01, 279]  - Score  0.7419467359960644\n",
      "Current iter:  30  - Best Score  0.7369712456359043  - Best Args:  [2, 2, 1, 2, 3, 3, 2, 2, 3, 2, 5, 219, 89, 246, 175, 79, <function relu at 0x79322f471900>, 0.04619831589851348, True, 0.001, 265]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 2, 'cat2_dim': 1, 'cat3_dim': 3, 'cat4_dim': 3, 'cat5_dim': 2, 'cat6_dim': 3, 'cat7_dim': 2, 'cat8_dim': 5, 'cat9_dim': 5, 'layers': 2, 'layer_1': 62, 'layer_2': 65, 'layer_3': 4, 'layer_4': 61, 'layer_5': 195, 'activation': <function leaky_relu at 0x79322f30e200>, 'dropout': 0.09561750243591474, 'batch_normalization': False, 'learning_rate': 0.005, 'batch_size': 1007}\n",
      "CV Fold 1 rmse:0.74296 - 2 rounds - it took 55 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73917\n",
      "Last eval:  [2, 2, 1, 3, 3, 2, 3, 2, 5, 5, 2, 62, 65, 4, 61, 195, <function leaky_relu at 0x79322f30e200>, 0.09561750243591474, False, 0.005, 1007]  - Score  0.7429602120019376\n",
      "Current iter:  31  - Best Score  0.7369712456359043  - Best Args:  [2, 2, 1, 2, 3, 3, 2, 2, 3, 2, 5, 219, 89, 246, 175, 79, <function relu at 0x79322f471900>, 0.04619831589851348, True, 0.001, 265]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 2, 'cat2_dim': 2, 'cat3_dim': 3, 'cat4_dim': 3, 'cat5_dim': 1, 'cat6_dim': 1, 'cat7_dim': 1, 'cat8_dim': 3, 'cat9_dim': 2, 'layers': 3, 'layer_1': 33, 'layer_2': 83, 'layer_3': 256, 'layer_4': 219, 'layer_5': 133, 'activation': <function relu at 0x79322f471900>, 'dropout': 0.07589222678537134, 'batch_normalization': False, 'learning_rate': 0.001, 'batch_size': 256}\n",
      "CV Fold 1 rmse:0.74079 - 9 rounds - it took 98 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73917\n",
      "Last eval:  [2, 2, 2, 3, 3, 1, 1, 1, 3, 2, 3, 33, 83, 256, 219, 133, <function relu at 0x79322f471900>, 0.07589222678537134, False, 0.001, 256]  - Score  0.7407880592032599\n",
      "Current iter:  32  - Best Score  0.7369712456359043  - Best Args:  [2, 2, 1, 2, 3, 3, 2, 2, 3, 2, 5, 219, 89, 246, 175, 79, <function relu at 0x79322f471900>, 0.04619831589851348, True, 0.001, 265]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 2, 'cat2_dim': 1, 'cat3_dim': 1, 'cat4_dim': 3, 'cat5_dim': 3, 'cat6_dim': 1, 'cat7_dim': 2, 'cat8_dim': 5, 'cat9_dim': 5, 'layers': 5, 'layer_1': 48, 'layer_2': 42, 'layer_3': 256, 'layer_4': 140, 'layer_5': 32, 'activation': <function relu at 0x79322f471900>, 'dropout': 0.414144369678964, 'batch_normalization': False, 'learning_rate': 0.005, 'batch_size': 260}\n",
      "CV Fold 1 rmse:0.74138 - 7 rounds - it took 98 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73917\n",
      "Last eval:  [2, 2, 1, 1, 3, 3, 1, 2, 5, 5, 5, 48, 42, 256, 140, 32, <function relu at 0x79322f471900>, 0.414144369678964, False, 0.005, 260]  - Score  0.7413779508617793\n",
      "Current iter:  33  - Best Score  0.7369712456359043  - Best Args:  [2, 2, 1, 2, 3, 3, 2, 2, 3, 2, 5, 219, 89, 246, 175, 79, <function relu at 0x79322f471900>, 0.04619831589851348, True, 0.001, 265]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 1, 'cat2_dim': 1, 'cat3_dim': 2, 'cat4_dim': 2, 'cat5_dim': 2, 'cat6_dim': 3, 'cat7_dim': 3, 'cat8_dim': 5, 'cat9_dim': 2, 'layers': 1, 'layer_1': 2, 'layer_2': 36, 'layer_3': 256, 'layer_4': 240, 'layer_5': 7, 'activation': <function relu at 0x79322f471900>, 'dropout': 0.24420755713788972, 'batch_normalization': False, 'learning_rate': 0.002, 'batch_size': 256}\n",
      "CV Fold 1 rmse:0.74794 - 15 rounds - it took 134 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73917\n",
      "Last eval:  [2, 1, 1, 2, 2, 2, 3, 3, 5, 2, 1, 2, 36, 256, 240, 7, <function relu at 0x79322f471900>, 0.24420755713788972, False, 0.002, 256]  - Score  0.7479411617479694\n",
      "Current iter:  34  - Best Score  0.7369712456359043  - Best Args:  [2, 2, 1, 2, 3, 3, 2, 2, 3, 2, 5, 219, 89, 246, 175, 79, <function relu at 0x79322f471900>, 0.04619831589851348, True, 0.001, 265]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 2, 'cat2_dim': 2, 'cat3_dim': 3, 'cat4_dim': 1, 'cat5_dim': 2, 'cat6_dim': 1, 'cat7_dim': 2, 'cat8_dim': 3, 'cat9_dim': 3, 'layers': 4, 'layer_1': 108, 'layer_2': 248, 'layer_3': 28, 'layer_4': 138, 'layer_5': 216, 'activation': <function leaky_relu at 0x79322f30e200>, 'dropout': 0.3977859316990666, 'batch_normalization': False, 'learning_rate': 0.002, 'batch_size': 457}\n",
      "CV Fold 1 rmse:0.73980 - 17 rounds - it took 155 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73917\n",
      "Last eval:  [2, 2, 2, 3, 1, 2, 1, 2, 3, 3, 4, 108, 248, 28, 138, 216, <function leaky_relu at 0x79322f30e200>, 0.3977859316990666, False, 0.002, 457]  - Score  0.7398015082179715\n",
      "Current iter:  35  - Best Score  0.7369712456359043  - Best Args:  [2, 2, 1, 2, 3, 3, 2, 2, 3, 2, 5, 219, 89, 246, 175, 79, <function relu at 0x79322f471900>, 0.04619831589851348, True, 0.001, 265]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 2, 'cat2_dim': 1, 'cat3_dim': 2, 'cat4_dim': 2, 'cat5_dim': 3, 'cat6_dim': 3, 'cat7_dim': 4, 'cat8_dim': 5, 'cat9_dim': 2, 'layers': 2, 'layer_1': 155, 'layer_2': 98, 'layer_3': 68, 'layer_4': 125, 'layer_5': 26, 'activation': <function leaky_relu at 0x79322f30e200>, 'dropout': 0.4357396340911345, 'batch_normalization': True, 'learning_rate': 0.001, 'batch_size': 341}\n",
      "CV Fold 1 rmse:0.74039 - 8 rounds - it took 97 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73917\n",
      "Last eval:  [2, 2, 1, 2, 2, 3, 3, 4, 5, 2, 2, 155, 98, 68, 125, 26, <function leaky_relu at 0x79322f30e200>, 0.4357396340911345, True, 0.001, 341]  - Score  0.7403885576245064\n",
      "Current iter:  36  - Best Score  0.7369712456359043  - Best Args:  [2, 2, 1, 2, 3, 3, 2, 2, 3, 2, 5, 219, 89, 246, 175, 79, <function relu at 0x79322f471900>, 0.04619831589851348, True, 0.001, 265]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 2, 'cat2_dim': 2, 'cat3_dim': 3, 'cat4_dim': 2, 'cat5_dim': 1, 'cat6_dim': 3, 'cat7_dim': 2, 'cat8_dim': 6, 'cat9_dim': 3, 'layers': 3, 'layer_1': 58, 'layer_2': 199, 'layer_3': 73, 'layer_4': 186, 'layer_5': 183, 'activation': <function leaky_relu at 0x79322f30e200>, 'dropout': 0.010067173519433316, 'batch_normalization': False, 'learning_rate': 0.002, 'batch_size': 980}\n",
      "CV Fold 1 rmse:0.74065 - 10 rounds - it took 113 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73917\n",
      "Last eval:  [2, 2, 2, 3, 2, 1, 3, 2, 6, 3, 3, 58, 199, 73, 186, 183, <function leaky_relu at 0x79322f30e200>, 0.010067173519433316, False, 0.002, 980]  - Score  0.7406544228150714\n",
      "Current iter:  37  - Best Score  0.7369712456359043  - Best Args:  [2, 2, 1, 2, 3, 3, 2, 2, 3, 2, 5, 219, 89, 246, 175, 79, <function relu at 0x79322f471900>, 0.04619831589851348, True, 0.001, 265]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 1, 'cat2_dim': 1, 'cat3_dim': 3, 'cat4_dim': 2, 'cat5_dim': 2, 'cat6_dim': 2, 'cat7_dim': 2, 'cat8_dim': 4, 'cat9_dim': 5, 'layers': 5, 'layer_1': 60, 'layer_2': 139, 'layer_3': 251, 'layer_4': 148, 'layer_5': 26, 'activation': <function relu at 0x79322f471900>, 'dropout': 0.32178268884896283, 'batch_normalization': False, 'learning_rate': 0.002, 'batch_size': 846}\n",
      "CV Fold 1 rmse:0.75091 - 11 rounds - it took 125 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73917\n",
      "Last eval:  [2, 1, 1, 3, 2, 2, 2, 2, 4, 5, 5, 60, 139, 251, 148, 26, <function relu at 0x79322f471900>, 0.32178268884896283, False, 0.002, 846]  - Score  0.7509120241583161\n",
      "Current iter:  38  - Best Score  0.7369712456359043  - Best Args:  [2, 2, 1, 2, 3, 3, 2, 2, 3, 2, 5, 219, 89, 246, 175, 79, <function relu at 0x79322f471900>, 0.04619831589851348, True, 0.001, 265]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 1, 'cat2_dim': 1, 'cat3_dim': 1, 'cat4_dim': 1, 'cat5_dim': 2, 'cat6_dim': 2, 'cat7_dim': 3, 'cat8_dim': 3, 'cat9_dim': 8, 'layers': 1, 'layer_1': 155, 'layer_2': 250, 'layer_3': 2, 'layer_4': 177, 'layer_5': 201, 'activation': <function relu at 0x79322f471900>, 'dropout': 0.3126870243256651, 'batch_normalization': True, 'learning_rate': 0.01, 'batch_size': 448}\n",
      "CV Fold 1 rmse:0.74041 - 3 rounds - it took 60 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73917\n",
      "Last eval:  [2, 1, 1, 1, 1, 2, 2, 3, 3, 8, 1, 155, 250, 2, 177, 201, <function relu at 0x79322f471900>, 0.3126870243256651, True, 0.01, 448]  - Score  0.7404085845674899\n",
      "Current iter:  39  - Best Score  0.7369712456359043  - Best Args:  [2, 2, 1, 2, 3, 3, 2, 2, 3, 2, 5, 219, 89, 246, 175, 79, <function relu at 0x79322f471900>, 0.04619831589851348, True, 0.001, 265]\n",
      "\n",
      "Testing:  {'cat0_dim': 2, 'cat1_dim': 2, 'cat2_dim': 1, 'cat3_dim': 2, 'cat4_dim': 3, 'cat5_dim': 1, 'cat6_dim': 1, 'cat7_dim': 4, 'cat8_dim': 2, 'cat9_dim': 2, 'layers': 3, 'layer_1': 203, 'layer_2': 33, 'layer_3': 256, 'layer_4': 72, 'layer_5': 27, 'activation': <function relu at 0x79322f471900>, 'dropout': 0.37086700455227306, 'batch_normalization': True, 'learning_rate': 0.001, 'batch_size': 574}\n",
      "CV Fold 1 rmse:0.73961 - 18 rounds - it took 164 secs\n",
      "Early stopping for under-performing fold: threshold is 0.73917\n",
      "Last eval:  [2, 2, 1, 2, 3, 1, 1, 4, 2, 2, 3, 203, 33, 256, 72, 27, <function relu at 0x79322f471900>, 0.37086700455227306, True, 0.001, 574]  - Score  0.739613611363505\n",
      "Current iter:  40  - Best Score  0.7369712456359043  - Best Args:  [2, 2, 1, 2, 3, 3, 2, 2, 3, 2, 5, 219, 89, 246, 175, 79, <function relu at 0x79322f471900>, 0.04619831589851348, True, 0.001, 265]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "64128"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x0, y0 = joblib.load('checkpoint.pkl')\n",
    "\n",
    "gp_round = gp_minimize(func=objective,\n",
    "                       x0=x0,              # already examined values for x\n",
    "                       y0=y0,              # observed values for x0\n",
    "                       dimensions=space,\n",
    "                       acq_func='gp_hedge',\n",
    "                       n_calls=30,\n",
    "                       n_initial_points=0,\n",
    "                       callback=[onstep],\n",
    "                       random_state=0)\n",
    "\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "665b445c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-06T01:55:34.931808Z",
     "iopub.status.busy": "2024-10-06T01:55:34.930954Z",
     "iopub.status.idle": "2024-10-06T01:55:34.946614Z",
     "shell.execute_reply": "2024-10-06T01:55:34.945515Z"
    },
    "papermill": {
     "duration": 0.034085,
     "end_time": "2024-10-06T01:55:34.948601",
     "exception": false,
     "start_time": "2024-10-06T01:55:34.914516",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40\n"
     ]
    }
   ],
   "source": [
    "x0, y0 = joblib.load('checkpoint.pkl')\n",
    "print(len(x0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "170ee06c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-06T01:55:34.980866Z",
     "iopub.status.busy": "2024-10-06T01:55:34.980469Z",
     "iopub.status.idle": "2024-10-06T01:55:34.986811Z",
     "shell.execute_reply": "2024-10-06T01:55:34.985605Z"
    },
    "papermill": {
     "duration": 0.025128,
     "end_time": "2024-10-06T01:55:34.988946",
     "exception": false,
     "start_time": "2024-10-06T01:55:34.963818",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best score: 0.73697\n",
      "Best hyperparameters:\n",
      "cat0_dim                  : 2\n",
      "cat1_dim                  : 2\n",
      "cat2_dim                  : 1\n",
      "cat3_dim                  : 2\n",
      "cat4_dim                  : 3\n",
      "cat5_dim                  : 3\n",
      "cat6_dim                  : 2\n",
      "cat7_dim                  : 2\n",
      "cat8_dim                  : 3\n",
      "cat9_dim                  : 2\n",
      "layers                    : 5\n",
      "layer_1                   : 219\n",
      "layer_2                   : 89\n",
      "layer_3                   : 246\n",
      "layer_4                   : 175\n",
      "layer_5                   : 79\n",
      "activation                : <function relu at 0x79322f471900>\n",
      "dropout                   : 0.04619831589851348\n",
      "batch_normalization       : True\n",
      "learning_rate             : 0.001\n",
      "batch_size                : 265\n"
     ]
    }
   ],
   "source": [
    "print(f\"Best score: {gp_round.fun:0.5f}\")\n",
    "print(\"Best hyperparameters:\")\n",
    "for sp, x in zip(gp_round.space, gp_round.x):\n",
    "    print(f\"{sp.name:25} : {x}\")"
   ]
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "gpu",
   "dataSources": [
    {
     "databundleVersionId": 2267921,
     "sourceId": 27423,
     "sourceType": "competition"
    }
   ],
   "isGpuEnabled": true,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 10246.650316,
   "end_time": "2024-10-06T01:55:38.362060",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2024-10-05T23:04:51.711744",
   "version": "2.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
